{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ! pip install steppy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import os\n",
    "import random\n",
    "import sys\n",
    "import multiprocessing as mp\n",
    "from functools import reduce\n",
    "\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import yaml\n",
    "from attrdict import AttrDict\n",
    "from functools import partial\n",
    "\n",
    "def create_submission(meta, predictions):\n",
    "    submission = pd.DataFrame({'SK_ID_CURR': meta['SK_ID_CURR'].tolist(),\n",
    "                               'TARGET': predictions\n",
    "                               })\n",
    "    return submission\n",
    "\n",
    "\n",
    "def verify_submission(submission, sample_submission):\n",
    "    assert submission.shape == sample_submission.shape, \\\n",
    "        'Expected submission to have shape {} but got {}'.format(sample_submission.shape, submission.shape)\n",
    "\n",
    "    for submission_id, correct_id in zip(submission['SK_ID_CURR'].values, sample_submission['SK_ID_CURR'].values):\n",
    "        assert correct_id == submission_id, \\\n",
    "            'Wrong id: expected {} but got {}'.format(correct_id, submission_id)\n",
    "\n",
    "\n",
    "def get_logger():\n",
    "    return logging.getLogger('home-credit')\n",
    "\n",
    "\n",
    "def init_logger():\n",
    "    logger = logging.getLogger('home-credit')\n",
    "    logger.setLevel(logging.INFO)\n",
    "    message_format = logging.Formatter(fmt='%(asctime)s %(name)s >>> %(message)s',\n",
    "                                       datefmt='%Y-%m-%d %H-%M-%S')\n",
    "\n",
    "    # console handler for validation info\n",
    "    ch_va = logging.StreamHandler(sys.stdout)\n",
    "    ch_va.setLevel(logging.INFO)\n",
    "\n",
    "    ch_va.setFormatter(fmt=message_format)\n",
    "\n",
    "    # add the handlers to the logger\n",
    "    logger.addHandler(ch_va)\n",
    "\n",
    "    return logger\n",
    "\n",
    "\n",
    "def read_params(ctx, fallback_file):\n",
    "    if ctx.params.__class__.__name__ == 'OfflineContextParams':\n",
    "        neptune_config = read_yaml(fallback_file)\n",
    "        params = neptune_config.parameters\n",
    "    else:\n",
    "        params = ctx.params\n",
    "    return params\n",
    "\n",
    "\n",
    "def read_yaml(filepath):\n",
    "    with open(filepath) as f:\n",
    "        config = yaml.load(f)\n",
    "    return AttrDict(config)\n",
    "\n",
    "\n",
    "def parameter_eval(param):\n",
    "    try:\n",
    "        return eval(param)\n",
    "    except Exception:\n",
    "        return param\n",
    "\n",
    "\n",
    "def persist_evaluation_predictions(experiment_directory, y_pred, raw_data, id_column, target_column):\n",
    "    raw_data.loc[:, 'y_pred'] = y_pred.reshape(-1)\n",
    "    predictions_df = raw_data.loc[:, [id_column, target_column, 'y_pred']]\n",
    "    filepath = os.path.join(experiment_directory, 'evaluation_predictions.csv')\n",
    "    logging.info('evaluation predictions csv shape: {}'.format(predictions_df.shape))\n",
    "    predictions_df.to_csv(filepath, index=None)\n",
    "\n",
    "\n",
    "def set_seed(seed=90210):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "\n",
    "def calculate_rank(predictions):\n",
    "    rank = (1 + predictions.rank().values) / (predictions.shape[0] + 1)\n",
    "    return rank\n",
    "\n",
    "\n",
    "def chunk_groups(groupby_object, chunk_size):\n",
    "    n_groups = groupby_object.ngroups\n",
    "    group_chunk, index_chunk = [], []\n",
    "    for i, (index, df) in enumerate(groupby_object):\n",
    "        group_chunk.append(df)\n",
    "        index_chunk.append(index)\n",
    "\n",
    "        if (i + 1) % chunk_size == 0 or i + 1 == n_groups:\n",
    "            group_chunk_, index_chunk_ = group_chunk.copy(), index_chunk.copy()\n",
    "            group_chunk, index_chunk = [], []\n",
    "            yield index_chunk_, group_chunk_\n",
    "\n",
    "\n",
    "def parallel_apply(groups, func, index_name='Index', num_workers=1, chunk_size=100000):\n",
    "    n_chunks = np.ceil(1.0 * groups.ngroups / chunk_size)\n",
    "    indeces, features = [], []\n",
    "    for index_chunk, groups_chunk in tqdm(chunk_groups(groups, chunk_size), total=n_chunks):\n",
    "        with mp.Pool(num_workers) as executor:\n",
    "            features_chunk = executor.map(func, groups_chunk)\n",
    "        features.extend(features_chunk)\n",
    "        indeces.extend(index_chunk)\n",
    "\n",
    "    features = pd.DataFrame(features)\n",
    "    features.index = indeces\n",
    "    features.index.name = index_name\n",
    "    return features\n",
    "\n",
    "\n",
    "def read_oof_predictions(prediction_dir, train_filepath, id_column, target_column):\n",
    "    labels = pd.read_csv(train_filepath, usecols=[id_column, target_column])\n",
    "\n",
    "    filepaths_train, filepaths_test = [], []\n",
    "    for filepath in sorted(glob.glob('{}/*'.format(prediction_dir))):\n",
    "        if filepath.endswith('_oof_train.csv'):\n",
    "            filepaths_train.append(filepath)\n",
    "        elif filepath.endswith('_oof_test.csv'):\n",
    "            filepaths_test.append(filepath)\n",
    "\n",
    "    train_dfs = []\n",
    "    for filepath in filepaths_train:\n",
    "        train_dfs.append(pd.read_csv(filepath))\n",
    "    train_dfs = reduce(lambda df1, df2: pd.merge(df1, df2, on=[id_column, 'fold_id']), train_dfs)\n",
    "    train_dfs.columns = _clean_columns(train_dfs, keep_colnames=[id_column, 'fold_id'])\n",
    "    train_dfs = pd.merge(train_dfs, labels, on=[id_column])\n",
    "\n",
    "    test_dfs = []\n",
    "    for filepath in filepaths_test:\n",
    "        test_dfs.append(pd.read_csv(filepath))\n",
    "    test_dfs = reduce(lambda df1, df2: pd.merge(df1, df2, on=[id_column, 'fold_id']), test_dfs)\n",
    "    test_dfs.columns = _clean_columns(test_dfs, keep_colnames=[id_column, 'fold_id'])\n",
    "    return train_dfs, test_dfs\n",
    "\n",
    "\n",
    "def _clean_columns(df, keep_colnames):\n",
    "    new_colnames = keep_colnames\n",
    "    feature_colnames = df.drop(keep_colnames, axis=1).columns\n",
    "    for i, colname in enumerate(feature_colnames):\n",
    "        new_colnames.append('model_{}'.format(i))\n",
    "    return new_colnames\n",
    "\n",
    "def safe_div(a, b):\n",
    "    try:\n",
    "        return float(a) / float(b)\n",
    "    except:\n",
    "        return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/sklearn/externals/joblib/_multiprocessing_helpers.py:28: UserWarning: [Errno 13] Permission denied.  joblib will operate in serial mode\n",
      "  warnings.warn('%s.  joblib will operate in serial mode' % (e,))\n"
     ]
    }
   ],
   "source": [
    "from steppy.base import BaseTransformer\n",
    "\n",
    "class BasicHandCraftedFeatures(BaseTransformer):\n",
    "    def __init__(self, num_workers=1, **kwargs):\n",
    "        self.num_workers = num_workers\n",
    "        self.features = None\n",
    "\n",
    "    @property\n",
    "    def feature_names(self):\n",
    "        feature_names = list(self.features.columns)\n",
    "        feature_names.remove('SK_ID_CURR')\n",
    "        return feature_names\n",
    "\n",
    "    def transform(self, **kwargs):\n",
    "        return {'features_table': self.features}\n",
    "\n",
    "    def load(self, filepath):\n",
    "        self.features = joblib.load(filepath)\n",
    "        return self\n",
    "\n",
    "    def persist(self, filepath):\n",
    "        joblib.dump(self.features, filepath)\n",
    "\n",
    "\n",
    "class ApplicationFeatures(BaseTransformer):\n",
    "    def __init__(self, categorical_columns, numerical_columns):\n",
    "        self.categorical_columns = categorical_columns\n",
    "        self.numerical_columns = numerical_columns\n",
    "        self.engineered_numerical_columns = ['annuity_income_percentage',\n",
    "                                             'car_to_birth_ratio',\n",
    "                                             'car_to_employ_ratio',\n",
    "                                             'children_ratio',\n",
    "                                             'credit_to_annuity_ratio',\n",
    "                                             'credit_to_goods_ratio',\n",
    "                                             'credit_to_income_ratio',\n",
    "                                             'days_employed_percentage',\n",
    "                                             'income_credit_percentage',\n",
    "                                             'income_per_child',\n",
    "                                             'income_per_person',\n",
    "                                             'payment_rate',\n",
    "                                             'phone_to_birth_ratio',\n",
    "                                             'phone_to_employ_ratio',\n",
    "                                             'external_sources_weighted',\n",
    "                                             'external_sources_min',\n",
    "                                             'external_sources_max',\n",
    "                                             'external_sources_sum',\n",
    "                                             'external_sources_mean',\n",
    "                                             'external_sources_nanmedian',\n",
    "                                             'short_employment',\n",
    "                                             'young_age',\n",
    "                                             'cnt_non_child',\n",
    "                                             'child_to_non_child_ratio',\n",
    "                                             'income_per_non_child',\n",
    "                                             'credit_per_person',\n",
    "                                             'credit_per_child',\n",
    "                                             'credit_per_non_child',\n",
    "                                             ]\n",
    "\n",
    "    def transform(self, X, **kwargs):\n",
    "        X['annuity_income_percentage'] = X['AMT_ANNUITY'] / X['AMT_INCOME_TOTAL']\n",
    "        X['car_to_birth_ratio'] = X['OWN_CAR_AGE'] / X['DAYS_BIRTH']\n",
    "        X['car_to_employ_ratio'] = X['OWN_CAR_AGE'] / X['DAYS_EMPLOYED']\n",
    "        X['children_ratio'] = X['CNT_CHILDREN'] / X['CNT_FAM_MEMBERS']\n",
    "        X['credit_to_annuity_ratio'] = X['AMT_CREDIT'] / X['AMT_ANNUITY']\n",
    "        X['credit_to_goods_ratio'] = X['AMT_CREDIT'] / X['AMT_GOODS_PRICE']\n",
    "        X['credit_to_income_ratio'] = X['AMT_CREDIT'] / X['AMT_INCOME_TOTAL']\n",
    "        X['days_employed_percentage'] = X['DAYS_EMPLOYED'] / X['DAYS_BIRTH']\n",
    "        X['income_credit_percentage'] = X['AMT_INCOME_TOTAL'] / X['AMT_CREDIT']\n",
    "        X['income_per_child'] = X['AMT_INCOME_TOTAL'] / (1 + X['CNT_CHILDREN'])\n",
    "        X['income_per_person'] = X['AMT_INCOME_TOTAL'] / X['CNT_FAM_MEMBERS']\n",
    "        X['payment_rate'] = X['AMT_ANNUITY'] / X['AMT_CREDIT']\n",
    "        X['phone_to_birth_ratio'] = X['DAYS_LAST_PHONE_CHANGE'] / X['DAYS_BIRTH']\n",
    "        X['phone_to_employ_ratio'] = X['DAYS_LAST_PHONE_CHANGE'] / X['DAYS_EMPLOYED']\n",
    "        X['external_sources_weighted'] = X.EXT_SOURCE_1 * 2 + X.EXT_SOURCE_2 * 3 + X.EXT_SOURCE_3 * 4\n",
    "        X['cnt_non_child'] = X['CNT_FAM_MEMBERS'] - X['CNT_CHILDREN']\n",
    "        X['child_to_non_child_ratio'] = X['CNT_CHILDREN'] / X['cnt_non_child']\n",
    "        X['income_per_non_child'] = X['AMT_INCOME_TOTAL'] / X['cnt_non_child']\n",
    "        X['credit_per_person'] = X['AMT_CREDIT'] / X['CNT_FAM_MEMBERS']\n",
    "        X['credit_per_child'] = X['AMT_CREDIT'] / (1 + X['CNT_CHILDREN'])\n",
    "        X['credit_per_non_child'] = X['AMT_CREDIT'] / X['cnt_non_child']\n",
    "        for function_name in ['min', 'max', 'sum', 'mean', 'nanmedian']:\n",
    "            X['external_sources_{}'.format(function_name)] = eval('np.{}'.format(function_name))(\n",
    "                X[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']], axis=1)\n",
    "\n",
    "        X['short_employment'] = (X['DAYS_EMPLOYED'] < -2000).astype(int)\n",
    "        X['young_age'] = (X['DAYS_BIRTH'] < -14000).astype(int)\n",
    "\n",
    "        return {'numerical_features': X[self.engineered_numerical_columns + self.numerical_columns],\n",
    "                'categorical_features': X[self.categorical_columns]\n",
    "}\n",
    "\n",
    "class POSCASHBalanceFeatures(BasicHandCraftedFeatures):\n",
    "    def __init__(self, last_k_agg_periods, last_k_trend_periods, num_workers=1, **kwargs):\n",
    "        super().__init__(num_workers=num_workers)\n",
    "        self.last_k_agg_periods = last_k_agg_periods\n",
    "        self.last_k_trend_periods = last_k_trend_periods\n",
    "\n",
    "        self.num_workers = num_workers\n",
    "        self.features = None\n",
    "\n",
    "    def fit(self, pos_cash, **kwargs):\n",
    "        pos_cash['is_contract_status_completed'] = pos_cash['NAME_CONTRACT_STATUS'] == 'Completed'\n",
    "        pos_cash['pos_cash_paid_late'] = (pos_cash['SK_DPD'] > 0).astype(int)\n",
    "        pos_cash['pos_cash_paid_late_with_tolerance'] = (pos_cash['SK_DPD_DEF'] > 0).astype(int)\n",
    "\n",
    "        features = pd.DataFrame({'SK_ID_CURR': pos_cash['SK_ID_CURR'].unique()})\n",
    "        groupby = pos_cash.groupby(['SK_ID_CURR'])\n",
    "        func = partial(POSCASHBalanceFeatures.generate_features,\n",
    "                       agg_periods=self.last_k_agg_periods,\n",
    "                       trend_periods=self.last_k_trend_periods)\n",
    "        g = parallel_apply(groupby, func, index_name='SK_ID_CURR', num_workers=self.num_workers).reset_index()\n",
    "        features = features.merge(g, on='SK_ID_CURR', how='left')\n",
    "\n",
    "        self.features = features\n",
    "        return self\n",
    "\n",
    "    @staticmethod\n",
    "    def generate_features(gr, agg_periods, trend_periods):\n",
    "        one_time = POSCASHBalanceFeatures.one_time_features(gr)\n",
    "        all = POSCASHBalanceFeatures.all_installment_features(gr)\n",
    "        agg = POSCASHBalanceFeatures.last_k_installment_features(gr, agg_periods)\n",
    "        trend = POSCASHBalanceFeatures.trend_in_last_k_installment_features(gr, trend_periods)\n",
    "        last = POSCASHBalanceFeatures.last_loan_features(gr)\n",
    "        features = {**one_time, **all, **agg, **trend, **last}\n",
    "        return pd.Series(features)\n",
    "\n",
    "    @staticmethod\n",
    "    def one_time_features(gr):\n",
    "        gr_ = gr.copy()\n",
    "        gr_.sort_values(['MONTHS_BALANCE'], inplace=True)\n",
    "        features = {}\n",
    "\n",
    "        features['pos_cash_remaining_installments'] = gr_['CNT_INSTALMENT_FUTURE'].tail(1)\n",
    "        features['pos_cash_completed_contracts'] = gr_['is_contract_status_completed'].agg('sum')\n",
    "\n",
    "        return features\n",
    "\n",
    "    @staticmethod\n",
    "    def all_installment_features(gr):\n",
    "        return POSCASHBalanceFeatures.last_k_installment_features(gr, periods=[10e16])\n",
    "\n",
    "    @staticmethod\n",
    "    def last_k_installment_features(gr, periods):\n",
    "        gr_ = gr.copy()\n",
    "        gr_.sort_values(['MONTHS_BALANCE'], ascending=False, inplace=True)\n",
    "\n",
    "        features = {}\n",
    "        for period in periods:\n",
    "            if period > 10e10:\n",
    "                period_name = 'all_installment_'\n",
    "                gr_period = gr_.copy()\n",
    "            else:\n",
    "                period_name = 'last_{}_'.format(period)\n",
    "                gr_period = gr_.iloc[:period]\n",
    "\n",
    "            features = add_features_in_group(features, gr_period, 'pos_cash_paid_late',\n",
    "                                             ['count', 'mean'],\n",
    "                                             period_name)\n",
    "            features = add_features_in_group(features, gr_period, 'pos_cash_paid_late_with_tolerance',\n",
    "                                             ['count', 'mean'],\n",
    "                                             period_name)\n",
    "            features = add_features_in_group(features, gr_period, 'SK_DPD',\n",
    "                                             ['sum', 'mean', 'max', 'std', 'skew', 'kurt'],\n",
    "                                             period_name)\n",
    "            features = add_features_in_group(features, gr_period, 'SK_DPD_DEF',\n",
    "                                             ['sum', 'mean', 'max', 'std', 'skew', 'kurt'],\n",
    "                                             period_name)\n",
    "        return features\n",
    "\n",
    "    @staticmethod\n",
    "    def trend_in_last_k_installment_features(gr, periods):\n",
    "        gr_ = gr.copy()\n",
    "        gr_.sort_values(['MONTHS_BALANCE'], ascending=False, inplace=True)\n",
    "\n",
    "        features = {}\n",
    "        for period in periods:\n",
    "            gr_period = gr_.iloc[:period]\n",
    "\n",
    "            features = add_trend_feature(features, gr_period,\n",
    "                                         'SK_DPD', '{}_period_trend_'.format(period)\n",
    "                                         )\n",
    "            features = add_trend_feature(features, gr_period,\n",
    "                                         'SK_DPD_DEF', '{}_period_trend_'.format(period)\n",
    "                                         )\n",
    "            features = add_trend_feature(features, gr_period,\n",
    "                                         'CNT_INSTALMENT_FUTURE', '{}_period_trend_'.format(period)\n",
    "                                         )\n",
    "        return features\n",
    "\n",
    "    @staticmethod\n",
    "    def last_loan_features(gr):\n",
    "        gr_ = gr.copy()\n",
    "        gr_.sort_values(['MONTHS_BALANCE'], ascending=False, inplace=True)\n",
    "        last_installment_id = gr_['SK_ID_PREV'].iloc[0]\n",
    "        gr_ = gr_[gr_['SK_ID_PREV'] == last_installment_id]\n",
    "\n",
    "        features={}\n",
    "        features = add_features_in_group(features, gr_, 'pos_cash_paid_late',\n",
    "                                         ['count', 'sum', 'mean'],\n",
    "                                         'last_loan_')\n",
    "        features = add_features_in_group(features, gr_, 'pos_cash_paid_late_with_tolerance',\n",
    "                                         ['mean'],\n",
    "                                         'last_loan_')\n",
    "        features = add_features_in_group(features, gr_, 'SK_DPD',\n",
    "                                         ['sum', 'mean', 'max', 'std'],\n",
    "                                         'last_loan_')\n",
    "        features = add_features_in_group(features, gr_, 'SK_DPD_DEF',\n",
    "                                         ['sum', 'mean', 'max', 'std'],\n",
    "                                         'last_loan_')\n",
    "\n",
    "        return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/joblib/_multiprocessing_helpers.py:28: UserWarning: [Errno 13] Permission denied.  joblib will operate in serial mode\n",
      "  warnings.warn('%s.  joblib will operate in serial mode' % (e,))\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import time\n",
    "import category_encoders as ce\n",
    "from contextlib import contextmanager\n",
    "import lightgbm as lgb\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "from scipy.cluster.vq import kmeans2, whiten\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn.decomposition import truncated_svd\n",
    "import category_encoders as ce\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn import preprocessing\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "num_rows = None\n",
    "EPS = 1e-100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "descretize = lambda x, n: list(map(str, list(pd.qcut(x, n, duplicates='drop'))))\n",
    "\n",
    "def binary_encoder(df, n_train):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    enc = ce.BinaryEncoder(impute_missing=True, cols=categorical_columns).fit(df[0:n_train], df[0:n_train]['TARGET'])\n",
    "    df = enc.transform(df)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df[new_columns]\n",
    "\n",
    "def helmert_encoder(df, n_train):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    enc = ce.HelmertEncoder(impute_missing=True, cols=categorical_columns).fit(df[0:n_train], df[0:n_train]['TARGET'])\n",
    "    df = enc.transform(df)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df[new_columns]\n",
    "\n",
    "def target_encoder(df, n_train):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    enc = ce.TargetEncoder(impute_missing=True, cols=categorical_columns).fit(df[0:n_train], df[0:n_train]['TARGET'])\n",
    "    df = enc.transform(df)\n",
    "    return df[categorical_columns]\n",
    "\n",
    "def poly_encoder(df, n_train):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    enc = ce.PolynomialEncoder(impute_missing=True, cols=categorical_columns).fit(df[0:n_train], df[0:n_train]['TARGET'])\n",
    "    df = enc.transform(df)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df[new_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = pd.read_csv('/media/limbo/Home-Credit/data/application_train.csv.zip', nrows=num_rows)\n",
    "# n_train = df.shape[0]\n",
    "\n",
    "# test_df = pd.read_csv('/media/limbo/Home-Credit/data/application_test.csv.zip', nrows=num_rows)\n",
    "# df = df.append(test_df).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#target_encoder(df, n_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# binary_encoder(df, n_train).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('../data/application_train.csv.zip', nrows=num_rows)\n",
    "n_train = train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def application_train_test(num_rows=num_rows, nan_as_category=False):\n",
    "    # Read data and merge\n",
    "    df = pd.read_csv('../data/application_train.csv.zip', nrows=num_rows)\n",
    "\n",
    "    n_train = df.shape[0]\n",
    "\n",
    "    test_df = pd.read_csv('../data/application_test.csv.zip', nrows=num_rows)\n",
    "    print(\"Train samples: {}, test samples: {}\".format(len(df), len(test_df)))\n",
    "    df = df.append(test_df).reset_index()\n",
    "\n",
    "    df['CODE_GENDER'].replace('XNA', np.nan, inplace=True)\n",
    "    df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace=True)\n",
    "    df['NAME_FAMILY_STATUS'].replace('Unknown', np.nan, inplace=True)\n",
    "    df['ORGANIZATION_TYPE'].replace('XNA', np.nan, inplace=True)\n",
    "    \n",
    "    df['annuity_income_percentage'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']\n",
    "    df['car_to_birth_ratio'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']\n",
    "    df['car_to_employ_ratio'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']\n",
    "    df['children_ratio'] = df['CNT_CHILDREN'] / df['CNT_FAM_MEMBERS']\n",
    "    df['credit_to_annuity_ratio'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']\n",
    "    df['credit_to_goods_ratio'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']\n",
    "    df['credit_to_income_ratio'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']\n",
    "    df['days_employed_percentage'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']\n",
    "    df['income_credit_percentage'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']\n",
    "    df['income_per_child'] = df['AMT_INCOME_TOTAL'] / (1 + df['CNT_CHILDREN'])\n",
    "    df['income_per_person'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']\n",
    "    df['payment_rate'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']\n",
    "    df['phone_to_birth_ratio'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']\n",
    "    df['phone_to_employ_ratio'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']\n",
    "    df['external_sources_weighted'] = df.EXT_SOURCE_1 * 2 + df.EXT_SOURCE_2 * 3 + df.EXT_SOURCE_3 * 4\n",
    "    df['cnt_non_child'] = df['CNT_FAM_MEMBERS'] - df['CNT_CHILDREN']\n",
    "    df['child_to_non_child_ratio'] = df['CNT_CHILDREN'] / df['cnt_non_child']\n",
    "    df['income_per_non_child'] = df['AMT_INCOME_TOTAL'] / df['cnt_non_child']\n",
    "    df['credit_per_person'] = df['AMT_CREDIT'] / df['CNT_FAM_MEMBERS']\n",
    "    df['credit_per_child'] = df['AMT_CREDIT'] / (1 + df['CNT_CHILDREN'])\n",
    "    df['credit_per_non_child'] = df['AMT_CREDIT'] / df['cnt_non_child']\n",
    "    for function_name in ['min', 'max', 'sum', 'mean', 'nanmedian']:\n",
    "        df['external_sources_{}'.format(function_name)] = eval('np.{}'.format(function_name))(\n",
    "        df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']], axis=1)\n",
    "\n",
    "    df['short_employment'] = (df['DAYS_EMPLOYED'] < -2000).astype(int)\n",
    "    df['young_age'] = (df['DAYS_BIRTH'] < -14000).astype(int)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train samples: 307511, test samples: 48744\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/numpy/lib/function_base.py:4033: RuntimeWarning: All-NaN slice encountered\n",
      "  r = func(a, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "df = application_train_test(num_rows=num_rows, nan_as_category=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_columns = [col for col in df.columns if df[col].dtype == 'object']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>EMERGENCYSTATE_MODE</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>FONDKAPREMONT_MODE</th>\n",
       "      <th>HOUSETYPE_MODE</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>NAME_EDUCATION_TYPE</th>\n",
       "      <th>NAME_FAMILY_STATUS</th>\n",
       "      <th>NAME_HOUSING_TYPE</th>\n",
       "      <th>NAME_INCOME_TYPE</th>\n",
       "      <th>NAME_TYPE_SUITE</th>\n",
       "      <th>OCCUPATION_TYPE</th>\n",
       "      <th>ORGANIZATION_TYPE</th>\n",
       "      <th>WALLSMATERIAL_MODE</th>\n",
       "      <th>WEEKDAY_APPR_PROCESS_START</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M</td>\n",
       "      <td>No</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>reg oper account</td>\n",
       "      <td>block of flats</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>Working</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>Business Entity Type 3</td>\n",
       "      <td>Stone, brick</td>\n",
       "      <td>WEDNESDAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>F</td>\n",
       "      <td>No</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>reg oper account</td>\n",
       "      <td>block of flats</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>Higher education</td>\n",
       "      <td>Married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>State servant</td>\n",
       "      <td>Family</td>\n",
       "      <td>Core staff</td>\n",
       "      <td>School</td>\n",
       "      <td>Block</td>\n",
       "      <td>MONDAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>Working</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>Government</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MONDAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Civil marriage</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>Working</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Laborers</td>\n",
       "      <td>Business Entity Type 3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WEDNESDAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>Secondary / secondary special</td>\n",
       "      <td>Single / not married</td>\n",
       "      <td>House / apartment</td>\n",
       "      <td>Working</td>\n",
       "      <td>Unaccompanied</td>\n",
       "      <td>Core staff</td>\n",
       "      <td>Religion</td>\n",
       "      <td>NaN</td>\n",
       "      <td>THURSDAY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  CODE_GENDER EMERGENCYSTATE_MODE FLAG_OWN_CAR FLAG_OWN_REALTY  \\\n",
       "0           M                  No            N               Y   \n",
       "1           F                  No            N               N   \n",
       "2           M                 NaN            Y               Y   \n",
       "3           F                 NaN            N               Y   \n",
       "4           M                 NaN            N               Y   \n",
       "\n",
       "  FONDKAPREMONT_MODE  HOUSETYPE_MODE NAME_CONTRACT_TYPE  \\\n",
       "0   reg oper account  block of flats         Cash loans   \n",
       "1   reg oper account  block of flats         Cash loans   \n",
       "2                NaN             NaN    Revolving loans   \n",
       "3                NaN             NaN         Cash loans   \n",
       "4                NaN             NaN         Cash loans   \n",
       "\n",
       "             NAME_EDUCATION_TYPE    NAME_FAMILY_STATUS  NAME_HOUSING_TYPE  \\\n",
       "0  Secondary / secondary special  Single / not married  House / apartment   \n",
       "1               Higher education               Married  House / apartment   \n",
       "2  Secondary / secondary special  Single / not married  House / apartment   \n",
       "3  Secondary / secondary special        Civil marriage  House / apartment   \n",
       "4  Secondary / secondary special  Single / not married  House / apartment   \n",
       "\n",
       "  NAME_INCOME_TYPE NAME_TYPE_SUITE OCCUPATION_TYPE       ORGANIZATION_TYPE  \\\n",
       "0          Working   Unaccompanied        Laborers  Business Entity Type 3   \n",
       "1    State servant          Family      Core staff                  School   \n",
       "2          Working   Unaccompanied        Laborers              Government   \n",
       "3          Working   Unaccompanied        Laborers  Business Entity Type 3   \n",
       "4          Working   Unaccompanied      Core staff                Religion   \n",
       "\n",
       "  WALLSMATERIAL_MODE WEEKDAY_APPR_PROCESS_START  \n",
       "0       Stone, brick                  WEDNESDAY  \n",
       "1              Block                     MONDAY  \n",
       "2                NaN                     MONDAY  \n",
       "3                NaN                  WEDNESDAY  \n",
       "4                NaN                   THURSDAY  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[categorical_columns].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 151)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encoder(df, nan_as_category=False):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df, new_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bureau_and_balance(num_rows = None, nan_as_category = True):\n",
    "    \n",
    "    bureau = pd.read_csv('../data/bureau.csv', nrows = num_rows)\n",
    "    bureau['bureau_credit_active_binary'] = (bureau['CREDIT_ACTIVE'] != 'Closed').astype(int)\n",
    "    bureau['bureau_credit_enddate_binary'] = (bureau['DAYS_CREDIT_ENDDATE'] > 0).astype(int)\n",
    "    features = pd.DataFrame({'SK_ID_CURR': bureau['SK_ID_CURR'].unique()})\n",
    "\n",
    "    groupby = bureau.groupby(by=['SK_ID_CURR'])\n",
    "\n",
    "    g = groupby['DAYS_CREDIT'].agg('count').reset_index()\n",
    "    g.rename(index=str, columns={'DAYS_CREDIT': 'bureau_number_of_past_loans'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['CREDIT_TYPE'].agg('nunique').reset_index()\n",
    "    g.rename(index=str, columns={'CREDIT_TYPE': 'bureau_number_of_loan_types'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['bureau_credit_active_binary'].agg('mean').reset_index()\n",
    "    g.rename(index=str, columns={'bureau_credit_active_binary': 'bureau_credit_active_binary'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['AMT_CREDIT_SUM_DEBT'].agg('sum').reset_index()\n",
    "    g.rename(index=str, columns={'AMT_CREDIT_SUM_DEBT': 'bureau_total_customer_debt'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['AMT_CREDIT_SUM'].agg('sum').reset_index()\n",
    "    g.rename(index=str, columns={'AMT_CREDIT_SUM': 'bureau_total_customer_credit'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['AMT_CREDIT_SUM_OVERDUE'].agg('sum').reset_index()\n",
    "    g.rename(index=str, columns={'AMT_CREDIT_SUM_OVERDUE': 'bureau_total_customer_overdue'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['CNT_CREDIT_PROLONG'].agg('sum').reset_index()\n",
    "    g.rename(index=str, columns={'CNT_CREDIT_PROLONG': 'bureau_average_creditdays_prolonged'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['bureau_credit_enddate_binary'].agg('mean').reset_index()\n",
    "    g.rename(index=str, columns={'bureau_credit_enddate_binary': 'bureau_credit_enddate_percentage'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    features['bureau_average_of_past_loans_per_type'] = \\\n",
    "        features['bureau_number_of_past_loans'] / features['bureau_number_of_loan_types']\n",
    "\n",
    "    features['bureau_debt_credit_ratio'] = \\\n",
    "        features['bureau_total_customer_debt'] / features['bureau_total_customer_credit']\n",
    "\n",
    "    features['bureau_overdue_debt_ratio'] = \\\n",
    "    features['bureau_total_customer_overdue'] / features['bureau_total_customer_debt']\n",
    "    \n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "bureau = bureau_and_balance(num_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(305811, 12)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bureau.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>bureau_number_of_past_loans</th>\n",
       "      <th>bureau_number_of_loan_types</th>\n",
       "      <th>bureau_credit_active_binary</th>\n",
       "      <th>bureau_total_customer_debt</th>\n",
       "      <th>bureau_total_customer_credit</th>\n",
       "      <th>bureau_total_customer_overdue</th>\n",
       "      <th>bureau_average_creditdays_prolonged</th>\n",
       "      <th>bureau_credit_enddate_percentage</th>\n",
       "      <th>bureau_average_of_past_loans_per_type</th>\n",
       "      <th>bureau_debt_credit_ratio</th>\n",
       "      <th>bureau_overdue_debt_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>215354</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>284463.18</td>\n",
       "      <td>5973945.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>3.666667</td>\n",
       "      <td>0.047617</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>162297</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>8230386.15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>402440</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>76905.00</td>\n",
       "      <td>89910.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.855355</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>238881</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>552730.50</td>\n",
       "      <td>1285239.06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.430060</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>222183</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>1185081.84</td>\n",
       "      <td>7158960.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>2.666667</td>\n",
       "      <td>0.165538</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  bureau_number_of_past_loans  bureau_number_of_loan_types  \\\n",
       "0      215354                           11                            3   \n",
       "1      162297                            6                            3   \n",
       "2      402440                            1                            1   \n",
       "3      238881                            8                            2   \n",
       "4      222183                            8                            3   \n",
       "\n",
       "   bureau_credit_active_binary  bureau_total_customer_debt  \\\n",
       "0                     0.545455                   284463.18   \n",
       "1                     0.500000                        0.00   \n",
       "2                     1.000000                    76905.00   \n",
       "3                     0.375000                   552730.50   \n",
       "4                     0.625000                  1185081.84   \n",
       "\n",
       "   bureau_total_customer_credit  bureau_total_customer_overdue  \\\n",
       "0                    5973945.30                            0.0   \n",
       "1                    8230386.15                            0.0   \n",
       "2                      89910.00                            0.0   \n",
       "3                    1285239.06                            0.0   \n",
       "4                    7158960.00                            0.0   \n",
       "\n",
       "   bureau_average_creditdays_prolonged  bureau_credit_enddate_percentage  \\\n",
       "0                                    0                          0.454545   \n",
       "1                                    0                          0.166667   \n",
       "2                                    0                          1.000000   \n",
       "3                                    0                          0.125000   \n",
       "4                                    0                          0.625000   \n",
       "\n",
       "   bureau_average_of_past_loans_per_type  bureau_debt_credit_ratio  \\\n",
       "0                               3.666667                  0.047617   \n",
       "1                               2.000000                  0.000000   \n",
       "2                               1.000000                  0.855355   \n",
       "3                               4.000000                  0.430060   \n",
       "4                               2.666667                  0.165538   \n",
       "\n",
       "   bureau_overdue_debt_ratio  \n",
       "0                        0.0  \n",
       "1                        NaN  \n",
       "2                        0.0  \n",
       "3                        0.0  \n",
       "4                        0.0  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bureau.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df[bureau.columns].to_csv('../data/neptune_bureau_0.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 218)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "189"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.join(bureau, how='left', on='SK_ID_CURR', lsuffix='_left', rsuffix='_right')\n",
    "# del bureau\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['SK_ID_CURR'] = df['SK_ID_CURR_left'].values\n",
    "del df['SK_ID_CURR_left']\n",
    "del df['SK_ID_CURR_right']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_GOODS_PRICE</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>...</th>\n",
       "      <th>bureau_credit_active_binary</th>\n",
       "      <th>bureau_total_customer_debt</th>\n",
       "      <th>bureau_total_customer_credit</th>\n",
       "      <th>bureau_total_customer_overdue</th>\n",
       "      <th>bureau_average_creditdays_prolonged</th>\n",
       "      <th>bureau_credit_enddate_percentage</th>\n",
       "      <th>bureau_average_of_past_loans_per_type</th>\n",
       "      <th>bureau_debt_credit_ratio</th>\n",
       "      <th>bureau_overdue_debt_ratio</th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>351000.0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>765000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>1129500.0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>43393.5</td>\n",
       "      <td>1372036.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.031627</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>457915.5</td>\n",
       "      <td>1133982.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.403812</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>297000.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>77346.0</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.859400</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>373176.0</td>\n",
       "      <td>462150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.807478</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100007</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 162 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  AMT_ANNUITY  AMT_CREDIT  AMT_GOODS_PRICE  AMT_INCOME_TOTAL  \\\n",
       "0      0      24700.5    406597.5         351000.0          202500.0   \n",
       "1      1      35698.5   1293502.5        1129500.0          270000.0   \n",
       "2      2       6750.0    135000.0         135000.0           67500.0   \n",
       "3      3      29686.5    312682.5         297000.0          135000.0   \n",
       "4      4      21865.5    513000.0         513000.0          121500.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_DAY  AMT_REQ_CREDIT_BUREAU_HOUR  \\\n",
       "0                        0.0                         0.0   \n",
       "1                        0.0                         0.0   \n",
       "2                        0.0                         0.0   \n",
       "3                        NaN                         NaN   \n",
       "4                        0.0                         0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_MON  AMT_REQ_CREDIT_BUREAU_QRT  \\\n",
       "0                        0.0                        0.0   \n",
       "1                        0.0                        0.0   \n",
       "2                        0.0                        0.0   \n",
       "3                        NaN                        NaN   \n",
       "4                        0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_WEEK    ...      bureau_credit_active_binary  \\\n",
       "0                         0.0    ...                         1.000000   \n",
       "1                         0.0    ...                         0.250000   \n",
       "2                         0.0    ...                         0.333333   \n",
       "3                         NaN    ...                         1.000000   \n",
       "4                         0.0    ...                         1.000000   \n",
       "\n",
       "   bureau_total_customer_debt  bureau_total_customer_credit  \\\n",
       "0                         0.0                      765000.0   \n",
       "1                     43393.5                     1372036.5   \n",
       "2                    457915.5                     1133982.0   \n",
       "3                     77346.0                       90000.0   \n",
       "4                    373176.0                      462150.0   \n",
       "\n",
       "   bureau_total_customer_overdue  bureau_average_creditdays_prolonged  \\\n",
       "0                            0.0                                  0.0   \n",
       "1                            0.0                                  0.0   \n",
       "2                            0.0                                  0.0   \n",
       "3                            0.0                                  0.0   \n",
       "4                            0.0                                  0.0   \n",
       "\n",
       "   bureau_credit_enddate_percentage  bureau_average_of_past_loans_per_type  \\\n",
       "0                          1.000000                                    1.0   \n",
       "1                          0.375000                                    8.0   \n",
       "2                          0.333333                                    6.0   \n",
       "3                          1.000000                                    1.0   \n",
       "4                          1.000000                                    1.0   \n",
       "\n",
       "   bureau_debt_credit_ratio  bureau_overdue_debt_ratio SK_ID_CURR  \n",
       "0                  0.000000                        NaN     100002  \n",
       "1                  0.031627                        0.0     100003  \n",
       "2                  0.403812                        0.0     100004  \n",
       "3                  0.859400                        0.0     100006  \n",
       "4                  0.807478                        0.0     100007  \n",
       "\n",
       "[5 rows x 162 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def previous_applications(num_rows=None, nan_as_category=True):\n",
    "    prev_applications = pd.read_csv('../data/previous_application.csv', nrows = num_rows)\n",
    "    \n",
    "    features = pd.DataFrame({'SK_ID_CURR': prev_applications['SK_ID_CURR'].unique()})\n",
    "\n",
    "    prev_app_sorted = prev_applications.sort_values(['SK_ID_CURR', 'DAYS_DECISION'])\n",
    "    prev_app_sorted_groupby = prev_app_sorted.groupby(by=['SK_ID_CURR'])\n",
    "\n",
    "    prev_app_sorted['previous_application_prev_was_approved'] = (\n",
    "        prev_app_sorted['NAME_CONTRACT_STATUS'] == 'Approved').astype('int')\n",
    "    g = prev_app_sorted_groupby['previous_application_prev_was_approved'].last().reset_index()\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    prev_app_sorted['previous_application_prev_was_refused'] = (\n",
    "        prev_app_sorted['NAME_CONTRACT_STATUS'] == 'Refused').astype('int')\n",
    "    g = prev_app_sorted_groupby['previous_application_prev_was_refused'].last().reset_index()\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = prev_app_sorted_groupby['SK_ID_PREV'].agg('nunique').reset_index()\n",
    "    g.rename(index=str, columns={'SK_ID_PREV': 'previous_application_number_of_prev_application'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = prev_app_sorted.groupby(by=['SK_ID_CURR'])['previous_application_prev_was_refused'].mean().reset_index()\n",
    "    g.rename(index=str, columns={\n",
    "        'previous_application_prev_was_refused': 'previous_application_fraction_of_refused_applications'},\n",
    "             inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    prev_app_sorted['prev_applications_prev_was_revolving_loan'] = (\n",
    "        prev_app_sorted['NAME_CONTRACT_TYPE'] == 'Revolving loans').astype('int')\n",
    "    g = prev_app_sorted.groupby(by=['SK_ID_CURR'])[\n",
    "        'prev_applications_prev_was_revolving_loan'].last().reset_index()\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    for number in [1, 2, 3, 8, 16, 32]:\n",
    "        prev_applications_tail = prev_app_sorted_groupby.tail(number)\n",
    "\n",
    "        tail_groupby = prev_applications_tail.groupby(by=['SK_ID_CURR'])\n",
    "\n",
    "        g = tail_groupby['CNT_PAYMENT'].agg('mean').reset_index()\n",
    "        g.rename(index=str,\n",
    "                 columns={'CNT_PAYMENT': 'previous_application_term_of_last_{}_credits_mean'.format(number)},\n",
    "                 inplace=True)\n",
    "        features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "        g = tail_groupby['DAYS_DECISION'].agg('mean').reset_index()\n",
    "        g.rename(index=str,\n",
    "                 columns={'DAYS_DECISION': 'previous_application_days_decision_about_last_{}_credits_mean'.format(\n",
    "                     number)},\n",
    "                 inplace=True)\n",
    "        features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "        g = tail_groupby['DAYS_FIRST_DRAWING'].agg('mean').reset_index()\n",
    "        g.rename(index=str,\n",
    "                 columns={\n",
    "                     'DAYS_FIRST_DRAWING': 'previous_application_days_first_drawing_last_{}_credits_mean'.format(\n",
    "                         number)},\n",
    "                 inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "    return features\n",
    "\n",
    "# Preprocess POS_CASH_balance.csv\n",
    "def pos_cash(num_rows = None, nan_as_category = True):\n",
    "    pos = pd.read_csv('../data/POS_CASH_balance.csv', nrows=num_rows)\n",
    "    pos['is_contract_status_completed'] = pos['NAME_CONTRACT_STATUS'] == 'Completed'\n",
    "    pos['pos_cash_paid_late'] = (pos['SK_DPD'] > 0).astype(int)\n",
    "    pos['pos_cash_paid_late_with_tolerance'] = (pos['SK_DPD_DEF'] > 0).astype(int)\n",
    "\n",
    "    features = pd.DataFrame({'SK_ID_CURR': pos['SK_ID_CURR'].unique()})\n",
    "#     groupby = pos.groupby(['SK_ID_CURR'])\n",
    "#     func = partial(POSCASHBalanceFeatures.generate_features,\n",
    "#                        agg_periods=[15, 30, 60, 180],\n",
    "#                        trend_periods=[15, 30, 60, 180])\n",
    "#     g = parallel_apply(groupby, func, index_name='SK_ID_CURR', num_workers=4).reset_index()\n",
    "    \n",
    "#     features = features.merge(g, on='SK_ID_CURR', how='left')\n",
    "    return features\n",
    "\n",
    "# Preprocess installments_payments.csv\n",
    "def installments_payments(num_rows = None, nan_as_category = True):\n",
    "    installments = pd.read_csv('/media/limbo/Home-Credit/data/installments_payments.csv', nrows = num_rows)\n",
    "    \n",
    "    installments['installment_paid_late_in_days'] = installments['DAYS_ENTRY_PAYMENT'] - installments[\n",
    "            'DAYS_INSTALMENT']\n",
    "    installments['installment_paid_late'] = (installments['installment_paid_late_in_days'] > 0).astype(int)\n",
    "    installments['installment_paid_over_amount'] = installments['AMT_PAYMENT'] - installments['AMT_INSTALMENT']\n",
    "    installments['installment_paid_over'] = (installments['installment_paid_over_amount'] > 0).astype(int)\n",
    "\n",
    "    features = pd.DataFrame({'SK_ID_CURR': installments['SK_ID_CURR'].unique()})\n",
    "#     groupby = installments.groupby(['SK_ID_CURR'])\n",
    "\n",
    "#     func = partial(InstallmentPaymentsFeatures.generate_features,\n",
    "#                    agg_periods=self.last_k_agg_periods,\n",
    "#                    period_fractions=self.last_k_agg_period_fractions,\n",
    "#                    trend_periods=self.last_k_trend_periods)\n",
    "#     g = parallel_apply(groupby, func, index_name='SK_ID_CURR', num_workers=self.num_workers).reset_index()\n",
    "    #features = features.merge(g, on='SK_ID_CURR', how='left')\n",
    "    return features\n",
    "\n",
    "# Preprocess credit_card_balance.csv\n",
    "def credit_card_balance(num_rows = None, nan_as_category = True):\n",
    "    credit_card = pd.read_csv('../data/credit_card_balance.csv', nrows = num_rows)\n",
    "    credit_card['number_of_installments'] = credit_card.groupby(\n",
    "            by=['SK_ID_CURR', 'SK_ID_PREV'])['CNT_INSTALMENT_MATURE_CUM'].agg('max').reset_index()[\n",
    "            'CNT_INSTALMENT_MATURE_CUM']\n",
    "\n",
    "    credit_card['credit_card_max_loading_of_credit_limit'] = credit_card.groupby(\n",
    "        by=['SK_ID_CURR', 'SK_ID_PREV', 'AMT_CREDIT_LIMIT_ACTUAL']).apply(\n",
    "        lambda x: x.AMT_BALANCE.max() / x.AMT_CREDIT_LIMIT_ACTUAL.max()).reset_index()[0]\n",
    "\n",
    "    features = pd.DataFrame({'SK_ID_CURR': credit_card['SK_ID_CURR'].unique()})\n",
    "\n",
    "    groupby = credit_card.groupby(by=['SK_ID_CURR'])\n",
    "\n",
    "    g = groupby['SK_ID_PREV'].agg('nunique').reset_index()\n",
    "    g.rename(index=str, columns={'SK_ID_PREV': 'credit_card_number_of_loans'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['SK_DPD'].agg('mean').reset_index()\n",
    "    g.rename(index=str, columns={'SK_DPD': 'credit_card_average_of_days_past_due'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['AMT_DRAWINGS_ATM_CURRENT'].agg('sum').reset_index()\n",
    "    g.rename(index=str, columns={'AMT_DRAWINGS_ATM_CURRENT': 'credit_card_drawings_atm'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['AMT_DRAWINGS_CURRENT'].agg('sum').reset_index()\n",
    "    g.rename(index=str, columns={'AMT_DRAWINGS_CURRENT': 'credit_card_drawings_total'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['number_of_installments'].agg('sum').reset_index()\n",
    "    g.rename(index=str, columns={'number_of_installments': 'credit_card_total_installments'}, inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    g = groupby['credit_card_max_loading_of_credit_limit'].agg('mean').reset_index()\n",
    "    g.rename(index=str,\n",
    "             columns={'credit_card_max_loading_of_credit_limit': 'credit_card_avg_loading_of_credit_limit'},\n",
    "             inplace=True)\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "\n",
    "    features['credit_card_cash_card_ratio'] = features['credit_card_drawings_atm'] / features[\n",
    "        'credit_card_drawings_total']\n",
    "\n",
    "    features['credit_card_installments_per_loan'] = (\n",
    "        features['credit_card_total_installments'] / features['credit_card_number_of_loans'])\n",
    "\n",
    "    return features\n",
    "\n",
    "def credit_card_balance_dynamic(num_rows = None, nan_as_category = True):\n",
    "    credit_card = pd.read_csv('../data/credit_card_balance.csv', nrows = num_rows)\n",
    "    features = pd.DataFrame({'SK_ID_CURR': credit_card['SK_ID_CURR'].unique()})\n",
    "\n",
    "    credit_card_sorted = credit_card.sort_values(['SK_ID_CURR', 'MONTHS_BALANCE'])\n",
    "\n",
    "    groupby = credit_card_sorted.groupby(by=['SK_ID_CURR'])\n",
    "    credit_card_sorted['credit_card_monthly_diff'] = groupby['AMT_BALANCE'].diff()\n",
    "    groupby = credit_card_sorted.groupby(by=['SK_ID_CURR'])\n",
    "\n",
    "    g = groupby['credit_card_monthly_diff'].agg('mean').reset_index()\n",
    "    features = features.merge(g, on=['SK_ID_CURR'], how='left')\n",
    "    return features\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/ipykernel_launcher.py:133: RuntimeWarning: invalid value encountered in double_scalars\n",
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/ipykernel_launcher.py:133: RuntimeWarning: divide by zero encountered in double_scalars\n"
     ]
    }
   ],
   "source": [
    "cc = credit_card_balance(num_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.join(cc, how='left', on='SK_ID_CURR', lsuffix='_left', rsuffix='_right')\n",
    "# del bureau\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['SK_ID_CURR'] = df['SK_ID_CURR_left'].values\n",
    "del df['SK_ID_CURR_left']\n",
    "del df['SK_ID_CURR_right']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "cc_dynamic = credit_card_balance_dynamic(num_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "84"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.join(cc_dynamic, how='left', on='SK_ID_CURR', lsuffix='_left', rsuffix='_right')\n",
    "# del bureau\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['SK_ID_CURR'] = df['SK_ID_CURR_left'].values\n",
    "del df['SK_ID_CURR_left']\n",
    "del df['SK_ID_CURR_right']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if __name__ == '__main__':\n",
    "pos = pos_cash(num_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.join(pos, how='left', on='SK_ID_CURR', lsuffix='_left', rsuffix='_right')\n",
    "# del bureau\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['SK_ID_CURR'] = df['SK_ID_CURR_left'].values\n",
    "del df['SK_ID_CURR_left']\n",
    "del df['SK_ID_CURR_right']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "prev = previous_applications(num_rows)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df[prev.columns].to_csv('../data/previous_applications_0.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "327"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.join(prev, how='left', on='SK_ID_CURR', lsuffix='_left', rsuffix='_right')\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['SK_ID_CURR'] = df['SK_ID_CURR_left'].values\n",
    "del df['SK_ID_CURR_left']\n",
    "del df['SK_ID_CURR_right']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "275"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ins = installments_payments(num_rows)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.join(ins, how='left', on='SK_ID_CURR', lsuffix='_left', rsuffix='_right')\n",
    "# del bureau\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['SK_ID_CURR'] = df['SK_ID_CURR_left'].values\n",
    "del df['SK_ID_CURR_left']\n",
    "del df['SK_ID_CURR_right']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['SK_ID_CURR']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_GOODS_PRICE</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>...</th>\n",
       "      <th>previous_application_term_of_last_3_credits_mean</th>\n",
       "      <th>previous_application_days_decision_about_last_3_credits_mean</th>\n",
       "      <th>previous_application_term_of_last_8_credits_mean</th>\n",
       "      <th>previous_application_days_decision_about_last_8_credits_mean</th>\n",
       "      <th>previous_application_term_of_last_16_credits_mean</th>\n",
       "      <th>previous_application_days_decision_about_last_16_credits_mean</th>\n",
       "      <th>previous_application_term_of_last_32_credits_mean</th>\n",
       "      <th>previous_application_days_decision_about_last_32_credits_mean</th>\n",
       "      <th>previous_application_days_first_drawing_last_32_credits_mean</th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>351000.0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>-221.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>-393.875</td>\n",
       "      <td>16.0</td>\n",
       "      <td>-889.000000</td>\n",
       "      <td>12.0</td>\n",
       "      <td>-1197.421053</td>\n",
       "      <td>303954.833333</td>\n",
       "      <td>100002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>1129500.0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>22.0</td>\n",
       "      <td>-822.666667</td>\n",
       "      <td>19.200000</td>\n",
       "      <td>-943.600</td>\n",
       "      <td>19.2</td>\n",
       "      <td>-943.600000</td>\n",
       "      <td>19.2</td>\n",
       "      <td>-943.600000</td>\n",
       "      <td>365243.000000</td>\n",
       "      <td>100003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>-421.666667</td>\n",
       "      <td>25.714286</td>\n",
       "      <td>-1013.625</td>\n",
       "      <td>24.0</td>\n",
       "      <td>-1179.111111</td>\n",
       "      <td>24.0</td>\n",
       "      <td>-1179.111111</td>\n",
       "      <td>365243.000000</td>\n",
       "      <td>100004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>297000.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>-735.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>-735.000</td>\n",
       "      <td>6.0</td>\n",
       "      <td>-735.000000</td>\n",
       "      <td>6.0</td>\n",
       "      <td>-735.000000</td>\n",
       "      <td>365243.000000</td>\n",
       "      <td>100006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>-266.500000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>-266.500</td>\n",
       "      <td>10.0</td>\n",
       "      <td>-266.500000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>-266.500000</td>\n",
       "      <td>365243.000000</td>\n",
       "      <td>100007</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 189 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  AMT_ANNUITY  AMT_CREDIT  AMT_GOODS_PRICE  AMT_INCOME_TOTAL  \\\n",
       "0      0      24700.5    406597.5         351000.0          202500.0   \n",
       "1      1      35698.5   1293502.5        1129500.0          270000.0   \n",
       "2      2       6750.0    135000.0         135000.0           67500.0   \n",
       "3      3      29686.5    312682.5         297000.0          135000.0   \n",
       "4      4      21865.5    513000.0         513000.0          121500.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_DAY  AMT_REQ_CREDIT_BUREAU_HOUR  \\\n",
       "0                        0.0                         0.0   \n",
       "1                        0.0                         0.0   \n",
       "2                        0.0                         0.0   \n",
       "3                        NaN                         NaN   \n",
       "4                        0.0                         0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_MON  AMT_REQ_CREDIT_BUREAU_QRT  \\\n",
       "0                        0.0                        0.0   \n",
       "1                        0.0                        0.0   \n",
       "2                        0.0                        0.0   \n",
       "3                        NaN                        NaN   \n",
       "4                        0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_WEEK    ...      \\\n",
       "0                         0.0    ...       \n",
       "1                         0.0    ...       \n",
       "2                         0.0    ...       \n",
       "3                         NaN    ...       \n",
       "4                         0.0    ...       \n",
       "\n",
       "   previous_application_term_of_last_3_credits_mean  \\\n",
       "0                                              24.0   \n",
       "1                                              22.0   \n",
       "2                                              24.0   \n",
       "3                                               6.0   \n",
       "4                                              10.0   \n",
       "\n",
       "   previous_application_days_decision_about_last_3_credits_mean  \\\n",
       "0                                        -221.000000              \n",
       "1                                        -822.666667              \n",
       "2                                        -421.666667              \n",
       "3                                        -735.000000              \n",
       "4                                        -266.500000              \n",
       "\n",
       "   previous_application_term_of_last_8_credits_mean  \\\n",
       "0                                         30.000000   \n",
       "1                                         19.200000   \n",
       "2                                         25.714286   \n",
       "3                                          6.000000   \n",
       "4                                         10.000000   \n",
       "\n",
       "   previous_application_days_decision_about_last_8_credits_mean  \\\n",
       "0                                           -393.875              \n",
       "1                                           -943.600              \n",
       "2                                          -1013.625              \n",
       "3                                           -735.000              \n",
       "4                                           -266.500              \n",
       "\n",
       "   previous_application_term_of_last_16_credits_mean  \\\n",
       "0                                               16.0   \n",
       "1                                               19.2   \n",
       "2                                               24.0   \n",
       "3                                                6.0   \n",
       "4                                               10.0   \n",
       "\n",
       "   previous_application_days_decision_about_last_16_credits_mean  \\\n",
       "0                                        -889.000000               \n",
       "1                                        -943.600000               \n",
       "2                                       -1179.111111               \n",
       "3                                        -735.000000               \n",
       "4                                        -266.500000               \n",
       "\n",
       "   previous_application_term_of_last_32_credits_mean  \\\n",
       "0                                               12.0   \n",
       "1                                               19.2   \n",
       "2                                               24.0   \n",
       "3                                                6.0   \n",
       "4                                               10.0   \n",
       "\n",
       "   previous_application_days_decision_about_last_32_credits_mean  \\\n",
       "0                                       -1197.421053               \n",
       "1                                        -943.600000               \n",
       "2                                       -1179.111111               \n",
       "3                                        -735.000000               \n",
       "4                                        -266.500000               \n",
       "\n",
       "   previous_application_days_first_drawing_last_32_credits_mean SK_ID_CURR  \n",
       "0                                      303954.833333                100002  \n",
       "1                                      365243.000000                100003  \n",
       "2                                      365243.000000                100004  \n",
       "3                                      365243.000000                100006  \n",
       "4                                      365243.000000                100007  \n",
       "\n",
       "[5 rows x 189 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_path = \"Level_1_stack/test_lgbm_k_3.csv\"\n",
    "validation_file_path = 'Level_1_stack/validation_lgbm_k_3.csv'\n",
    "num_folds = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_features = pd.read_csv('selected_features.csv', header=0, index_col=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniuqe = [f for f in new_features.columns if f not in df.columns]\n",
    "uniuqe\n",
    "new_df = pd.concat([df, new_features[uniuqe]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 214)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_3_months_features = pd.read_csv('../data/last_3_months_payments.csv', header=0, index_col=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniuqe_0 = [f for f in last_3_months_features.columns if f not in new_df.columns]\n",
    "new_df = pd.concat([new_df, last_3_months_features[uniuqe_0]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 218)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting LightGBM. Train shape: (307511, 218), test shape: (48744, 218)\n",
      "(307511, 215)\n",
      "(246008, 364) (61503, 364) (48744, 364)\n",
      "Fold  1 AUC : 0.782377\n",
      "(246009, 363) (61502, 363) (48744, 363)\n",
      "Fold  2 AUC : 0.754661\n",
      "(246009, 364) (61502, 364) (48744, 364)\n",
      "Fold  3 AUC : 0.774387\n",
      "(246009, 364) (61502, 364) (48744, 364)\n",
      "Fold  4 AUC : 0.778663\n",
      "(246009, 364) (61502, 364) (48744, 364)\n",
      "Fold  5 AUC : 0.779519\n",
      "Full AUC score 0.770298\n"
     ]
    }
   ],
   "source": [
    "encoding = 'ohe'\n",
    "\n",
    "train_df = df.iloc[0:n_train]\n",
    "test_df = df.iloc[n_train:]\n",
    "\n",
    "print(\"Starting LightGBM. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n",
    "gc.collect()\n",
    "# Cross validation model\n",
    "folds = KFold(n_splits=num_folds, shuffle=True, random_state=1001)\n",
    "# Create arrays and dataframes to store results\n",
    "oof_preds = np.zeros(train_df.shape[0])\n",
    "sub_preds = np.zeros(test_df.shape[0])\n",
    "feature_importance_df = pd.DataFrame()\n",
    "feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]\n",
    "\n",
    "#feats = [col for col in feats_0 if df[col].dtype == 'object']\n",
    "\n",
    "\n",
    "print(train_df[feats].shape)\n",
    "for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train_df['TARGET'])):\n",
    "        \n",
    "      \n",
    "        categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "        \n",
    "        if encoding == 'ohe':\n",
    "            \n",
    "            enc = ce.OneHotEncoder(impute_missing=True, cols=categorical_columns).fit(train_df[feats].iloc[train_idx],\n",
    "                                                                                       train_df['TARGET'].iloc[train_idx])\n",
    "            x_train = enc.transform(train_df[feats].iloc[train_idx])\n",
    "            x_valid = enc.transform(train_df[feats].iloc[valid_idx])\n",
    "            x_test = enc.transform(test_df[feats])\n",
    "            print(x_train.shape, x_valid.shape, x_test.shape)\n",
    "            \n",
    "            dtrain = lgb.Dataset(data=x_train, \n",
    "                             label=train_df['TARGET'].iloc[train_idx], \n",
    "                             free_raw_data=False, silent=True)\n",
    "            dvalid = lgb.Dataset(data=x_valid, \n",
    "                             label=train_df['TARGET'].iloc[valid_idx], \n",
    "                             free_raw_data=False, silent=True)\n",
    "            gc.collect()\n",
    "            \n",
    "        params = {\n",
    "            'objective': 'regression',\n",
    "            'boosting_type': 'goss',\n",
    "            'nthread': 8,\n",
    "            'learning_rate': 0.02,  \n",
    "            'num_leaves': 2 ** 5,\n",
    "            'colsample_bytree': 0.25,\n",
    "            'subsample': 0.9,\n",
    "            'max_depth': 8,\n",
    "            'reg_alpha': 1,\n",
    "            'reg_lambda': 2 ** 5,\n",
    "            'min_split_gain': 0.1,\n",
    "            'min_child_weight': 2 ** 5,\n",
    "            'seed': 666,\n",
    "            'scale_pos_weight' : 3,\n",
    "            'verbose': -1,\n",
    "            'metric': 'rmse',\n",
    "            'max_bin': 2 ** 7\n",
    "        }\n",
    "        \n",
    "        clf = lgb.train(\n",
    "            params=params,\n",
    "            train_set=dtrain,\n",
    "            num_boost_round=10000,\n",
    "            valid_sets=[dtrain, dvalid],\n",
    "            early_stopping_rounds=100,\n",
    "            verbose_eval=False\n",
    "        )\n",
    "        \n",
    "        oof_preds[valid_idx] = clf.predict(dvalid.data)\n",
    "        sub_preds += clf.predict(x_test) / folds.n_splits\n",
    "\n",
    "        print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(dvalid.label, oof_preds[valid_idx])))\n",
    "        del clf, dtrain, dvalid\n",
    "        gc.collect()\n",
    "\n",
    "print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))\n",
    "        \n",
    "sub_df = test_df[['SK_ID_CURR']].copy()\n",
    "sub_df['TARGET'] = sub_preds\n",
    "sub_df[['SK_ID_CURR', 'TARGET']].to_csv(test_file_path, index= False)\n",
    "\n",
    "val_df = train_df[['SK_ID_CURR', 'TARGET']].copy()\n",
    "val_df['TARGET'] = oof_preds\n",
    "val_df[['SK_ID_CURR', 'TARGET']].to_csv(validation_file_path, index= False)        \n",
    "            \n",
    "            \n",
    "        \n",
    "        \n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
